
import os
import random
import asyncio
import time
from playwright.async_api import async_playwright
from setting import DEBUG_PRINT, REQUEST_INTERVAL
from until import dynamic_route, move_window_by_title, time_print
from until import getCategoryIndex, getPageUrl, getPageIndex, getPageID
from dumpCSV import MeishitianxiaCSV
from ContextPagePool import ContextPagePool
from redisModel import RedisClient
from taskLog import AsyncTaskLogger

class MeishitianxiaPlaywright:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    def __init__(self, param):

        self.param = param
        self.categoryRange = param['categoryRange']
        self.categoryFile = param['categoryFile']
        self.categoryDir = './CSV'
        self.max_retries = 3
        # self.DEBUG_PRINT = True
        self.playwright = None
        self.browser = None
        self.semaphore = asyncio.Semaphore(param['maxTask'])
        # self.init_playwright()
        self.dump = MeishitianxiaCSV(param['saveDir'])
        self.redis_cli = RedisClient.get_redis()

        self.categoryPageNum = {}

        
    async def init_playwright(self):
        """初始化 Playwright"""

        self.playwright = await async_playwright().start()
        self.browser = await self.playwright.chromium.launch(
            headless=False,
            args=["--no-sandbox",
                "--disable-blink-features=AutomationControlled",
                "--disable-dev-shm-usage",
                "--disable-gpu"
            ]
        )
        self.contextPagePool = ContextPagePool(self.browser, self.param['maxTask'], self.param['IPTimeOout'])
        await self.contextPagePool.init_pool()
    
    def readCategoryUrls(self, categoryFile):
        ret = []
        with open(categoryFile, 'r', encoding='utf-8') as f:
            for lineIndex, line in enumerate(f):
                cleaned_line = line.strip()
                if cleaned_line:
                    dishUrl = cleaned_line.split(',')[-1]  # 只分割第一个逗号
                    if not dishUrl.startswith('http'):
                        continue
                    ret.append(dishUrl)
        return ret
    
    def getRangeCSV(self):
        allCSV = [os.path.join(self.categoryDir, item) for item in os.listdir(self.categoryDir)]
        allCSV.sort()
        rangeCSV = allCSV[self.categoryRange[0]:self.categoryRange[1]]
        return rangeCSV
        
    async def processCategory(self):

        categoryURLs = self.readCategoryUrls(self.categoryFile)
        rangeCategoryURLs = categoryURLs[self.categoryRange[0]:self.categoryRange[1]]

        for _, oneCategoryURL in enumerate(rangeCategoryURLs):
            await self.processOneCategory(oneCategoryURL)

    async def isCategoryEnd(self, categoryIndex, pageIndex):

        if categoryIndex in self.categoryPageNum:
            if pageIndex >= self.categoryPageNum[categoryIndex]:
                return True
        return False

    async def processOneCategory(self, oneCategoryURL):
        
        tasks = []
        categoryIndex = await getCategoryIndex(oneCategoryURL)
        time_print(f"=== 开始处理分类 {categoryIndex} ===")
        
        task_count = 0
        for pageIndex in range(1, 201):
            pageUrl = await getPageUrl(oneCategoryURL, pageIndex)
            isExist = self.redis_cli.sismember(f'caipu:Meishitianxia:{categoryIndex}', pageUrl)
            if isExist:
                print(f'该dish page 已经被抓取过 {pageUrl}')
                continue
            tasks.append(self.processOnePageWithLimit(categoryIndex, pageUrl))
            task_count += 1
        
        time_print(f"分类 {categoryIndex} 创建了 {task_count} 个任务，开始 gather")
        time_print(f"分类 {categoryIndex} 当前 semaphore 可用数量: {self.semaphore._value}")
        
        try:
            results = await asyncio.gather(*tasks, return_exceptions=True)
            time_print(f"分类 {categoryIndex} gather 完成，结果数量: {len(results)}")
            
            # 检查是否有异常
            exceptions = [r for r in results if isinstance(r, Exception)]
            if exceptions:
                time_print(f"分类 {categoryIndex} 有 {len(exceptions)} 个异常:")
                for i, exc in enumerate(exceptions):
                    time_print(f"  异常 {i+1}: {type(exc).__name__}: {exc}")
            
            successful_tasks = len([r for r in results if not isinstance(r, Exception)])
            time_print(f"分类 {categoryIndex} 成功完成 {successful_tasks} 个任务")
            
        except Exception as e:
            time_print(f"分类 {categoryIndex} gather 本身异常: {type(e).__name__}: {e}")
            raise
        
        # 清理logger资源
        AsyncTaskLogger.cleanup_all_loggers()
        time_print("已清理所有logger资源")
        time_print(f"=== 分类 {categoryIndex} 处理完成 ===")


    def _generate_track(self, distance):
        """
        生成拟人滑动轨迹，返回[(x_offset, y_offset), ...]
        """
        track = []
        current = 0
        mid = distance * 3 / 5
        t = 0.5
        v = 0
        while current < distance:
            if current < mid:
                a = random.uniform(2, 4)
            else:
                a = -random.uniform(3, 5)
            v0 = v
            v = v0 + a * t
            move = v0 * t + 0.5 * a * (t ** 2)
            current += move
            track.append((round(current), random.randint(-3, 3)))
        return track



    async def _simulate_slider_move(self, task_id, page):
        
        if DEBUG_PRINT >= 1:
            logger = AsyncTaskLogger.get_logger(task_id)
            logger.info(f"_simulate_slider_move started...")

        try:
            if DEBUG_PRINT >= 2:
                msg=f"1@@@ start {page.url}"
                logger.debug(f'{msg}')

            # slider = await page.locator('#nc_1_n1z').wait_for(timeout=5000)
            # sliderCount = await page.locator('#nc_1_n1z').count()
            await page.wait_for_selector('#nc_1_n1z', state='attached', timeout=5000)
            box = await page.evaluate('''() => {
                const el = document.querySelector('#nc_1_n1z');
                if (!el) throw new Error("元素未找到");
                const rect = el.getBoundingClientRect();
                return { x: rect.x, y: rect.y, width: rect.width, height: rect.height };
            }''')
            start_x = box['x'] + box['width'] / 2
            start_y = box['y'] + box['height'] / 2
            end_x = start_x + 300

            #模拟人随即移动鼠标，骗过反爬器
            if DEBUG_PRINT >= 2:
                msg=f'2@@@ 模拟人随即移动鼠标，骗过反爬器 {page.url}'
                logger.debug(f'{msg}')
            await page.mouse.move(
                start_x + random.randint(-50, 50),
                start_y + random.randint(-50, 50),
                steps=5
            )

            # 生成带加减速的人类轨迹
            if DEBUG_PRINT >= 2:
                msg=f'3@@@ 生成带加减速的人类轨迹 {page.url}'
                logger.debug(f'{msg}')
            await page.mouse.move(start_x, start_y)
            await page.mouse.down()
            track = self._generate_track(end_x - start_x)
            for offset_x, offset_y in track:
                await page.mouse.move(start_x + offset_x, start_y + offset_y, steps=1)
                await asyncio.sleep(random.uniform(0.01, 0.03))
            await page.mouse.up()
            await asyncio.sleep(1)

            # 设置弹出对话框事件回调函数
            if DEBUG_PRINT >= 2:
                msg=f'4@@@ 设置弹出对话框事件回调函数 {page.url}'
                logger.debug(f'{msg}')
            async def handleDlg(dialog):
                # 等待1秒
                await page.wait_for_timeout(1000)
                # 点击确定
                await dialog.accept()
                # 打印 弹出框 提示信息
                print(dialog.message)

            page.on("dialog", handleDlg)
        except Exception as e:
            # await page.screenshot(path=f"./screenshot/debug_{time.time()}.png")
            if DEBUG_PRINT >= 1:
                msg=f'5@@@ 滑块验证 出现意外错误 {page.url}'
                # logger.debug(f'{msg}')
                AsyncTaskLogger.log_exception(e, task_id, msg)
            pass
        if DEBUG_PRINT >= 1:
            logger.info(f"_simulate_slider_move Completed")

    async def handleSlider(self, task_id, page):
        if DEBUG_PRINT >= 1:
            logger = AsyncTaskLogger.get_logger(task_id)
            logger.info(f"handleSlider started...")# 运行这个后，异常退出了

        retry_count = 1
        while retry_count <= self.max_retries:
            try:
                if DEBUG_PRINT >= 2:
                    msg = f'1$$$ {retry_count} 次 {page.url}'
                    logger.debug(f'{msg}')
                await self._simulate_slider_move(task_id, page)
                if DEBUG_PRINT >= 2:
                    msg = f'2$$$ {retry_count} {page.url}'
                    logger.debug(f'{msg}')
                if '滑动验证' not in await page.title():
                    if DEBUG_PRINT >= 2:
                        msg = f'3$$$ {retry_count} 次 {page.url} {await page.title()}'
                        logger.debug(f'{msg}')
                        logger.info(f"handleSlider Completed")
                    return
            except Exception as e:
                if DEBUG_PRINT >= 1:
                    msg = f'4$$$ 滑动出现意外 {retry_count} 次 {page.url} {e}'
                    AsyncTaskLogger.log_exception(e, task_id, msg)
                pass
            retry_count += 1
        if DEBUG_PRINT >= 1:
            logger.info(f"handleSlider Completed")

    async def isPageOK(self, task_id, page, response):
        if DEBUG_PRINT >= 1:
            logger = AsyncTaskLogger.get_logger(task_id)
            logger.info(f"isPageOK started...")

        t0_isPageOK = time.time()
        if DEBUG_PRINT >= 2:
            t1_isPageOK = time.time()
            msg = f'task_id={task_id} 0*** {response.url} time: {round(t1_isPageOK-t0_isPageOK, 3)}'
            logger.debug(f'{msg}')
        console_errors = []
        def console_handler(msg):
            if msg.type == 'error':
                console_errors.append(msg.text)
        page.on('console', console_handler)
        if DEBUG_PRINT >= 2:
            t2_isPageOK = time.time()
            msg = f'task_id={task_id} 1*** response.url {response.url} time: {round(t2_isPageOK-t1_isPageOK, 3)}'
            logger.debug(f'{msg}')
        if 'humancheck_captcha' in response.url:
            try:
                await page.unroute("**/*")
                await page.reload(wait_until="networkidle", timeout=5000)
                await page.wait_for_selector("#nc_1_n1z", state="attached", timeout=3000)
            except Exception as e:
                await page.route("**/*", dynamic_route)
                t3_1_isPageOK = time.time()
                if DEBUG_PRINT >= 1:
                    msg = f'task_id={task_id} 1_1***  reload {response.url} {e} time: {round(t3_1_isPageOK-t1_isPageOK, 3)}'
                    AsyncTaskLogger.log_exception(e, task_id, msg)
                return False

            if DEBUG_PRINT >= 2:
                t3_isPageOK = time.time()
                msg = f'task_id={task_id} 2*** 滑动验证 {response.url} time: {round(t3_isPageOK-t1_isPageOK, 3)}'
                logger.debug(f'{msg}')
            await self.handleSlider(task_id, page)
            await page.route("**/*", dynamic_route)
            if DEBUG_PRINT >= 2:
                t4_isPageOK = time.time()
                msg=f'task_id={task_id} 3*** 滑动验证完成 {response.url} time: {round(t4_isPageOK-t1_isPageOK, 3)}'
                logger.debug(f'{msg}')
                logger.info(f"isPageOK Completed")
            return False

        if not response or response.status != 200:
            if DEBUG_PRINT >= 2:
                t5_isPageOK = time.time()
                msg=f'task_id={task_id} 4*** response.status !=200 {response.url} time: {round(t5_isPageOK-t1_isPageOK, 3)}'
                logger.debug(f'{msg}')
                logger.info(f"isPageOK Completed")
            return False
        
        if console_errors:
            if DEBUG_PRINT >= 2:
                t6_isPageOK = time.time()
                msg=f'task_id={task_id} 5*** console error {response.url} time: {round(t6_isPageOK-t1_isPageOK, 3)}'
                logger.debug(f'{msg}')
                logger.info(f"isPageOK Completed")
            return False

        try:
            await page.wait_for_selector('.ft1>.c3b>a:has-text("美食天下 - 让吃更美好")', state="attached", timeout=3000)
            # await page.wait_for_selector('#steps:has-text("的做法") , .score.float-left > .title:has-text("综合评分")', state="attached", timeout=3000)
            if DEBUG_PRINT >= 2:
                t7_isPageOK = time.time()
                msg=f'task_id={task_id} 6*** 找到xxx菜单列表 {response.url} time: {round(t7_isPageOK-t1_isPageOK, 3)}'
                logger.debug(f'{msg}')
                logger.info(f"isPageOK Completed")
            return True
        except Exception as e:
            print(e)
            if DEBUG_PRINT >= 1:
                t8_isPageOK = time.time()
                msg = f'task_id={task_id} 7*** 没找到xxx菜单列表 {response.url} {e} time: {round(t8_isPageOK-t1_isPageOK, 3)}'
                AsyncTaskLogger.log_exception(e, task_id, msg)
                logger.info(f"isPageOK Completed")
            return False


    async def pageGo(self, task_id, page, url):
        if DEBUG_PRINT >= 1:
            logger = AsyncTaskLogger.get_logger(task_id)
            logger.info(f"pageGo started...")

        retryTimes = 3
        for i in range(retryTimes):
            t1_pageGo = time.time()
            try:
                response = await page.goto(url, wait_until="domcontentloaded", timeout=10000)
                if DEBUG_PRINT >= 2:
                    t2_pageGo = time.time()
                    msg = f'task_id={task_id} 1+++ {i} 次 {response.status} {page.url} time: {round(t2_pageGo-t1_pageGo, 3)}'
                    logger.debug(f"{msg}")
                if not await self.isPageOK(task_id, page, response):
                    if DEBUG_PRINT >= 2:
                        t3_pageGo = time.time()
                        msg = f'task_id={task_id} 2+++ {i} 次 {page.url} time: {round(t3_pageGo-t2_pageGo, 3)}'
                        logger.debug(f"{msg}")
                    continue
                if DEBUG_PRINT >= 1:
                    logger.info(f"pageGo Completed")
                return True
            except Exception as e:
                if DEBUG_PRINT >= 1:
                    t4_pageGo = time.time()
                    msg = f'task_id={task_id} 3+++ pageGo exception {i} 次 {page.url} time: {round(t4_pageGo-t1_pageGo, 3)} {e}'
                    AsyncTaskLogger.log_exception(e, task_id, msg)
                continue
        if DEBUG_PRINT >= 1:
            logger.info(f"pageGo Completed")
        return False


    async def extractOnePage(self, categoryIndex, page):
        # dishs = await page.locator('.ui_newlist_1.get_num > ul > li > div > h2 > a').all_text_contents()
        dishs = await page.locator('.ui_newlist_1.get_num > ul > li:not(.clear)').all()
        dishNames = [await dish.locator('.detail > h2 > a').text_content() for dish in dishs]
        dishImgs = [await dish.locator('.pic > a > img').get_attribute('data-src') for dish in dishs]
        dishLinks = [await dish.locator('.detail > h2 > a').get_attribute('href') for dish in dishs]

        # try:
        #     await page.wait_for_selector('.ui-page-inner>a:has-text("下一页")', state="attached", timeout=3000)
        # except:
        #     pass
        nextPage = await page.locator('.ui-page-inner>a:has-text("下一页")').count()
        if not (nextPage > 0) and 'page' in page.url and (categoryIndex not in self.categoryPageNum):
            self.categoryPageNum[categoryIndex] = await getPageIndex(page.url)
        ret = {}
        ret['dish_names'] = dishNames
        ret['dish_imgs'] = dishImgs
        ret['dish_links'] = dishLinks
        ret['page_url'] = page.url
        return ret

    async def processOnePage(self, categoryIndex, pageUrl):
        
        pageID = await getPageID(categoryIndex, pageUrl)
        if DEBUG_PRINT >= 1:
            AsyncTaskLogger.log_task_start(task_id=pageID)
            logger = AsyncTaskLogger.get_logger(task_id=pageID)
            logger.info(f"processOneDish Starting processing...")

        pageIndex = await getPageIndex(pageUrl)
        if await self.isCategoryEnd(categoryIndex, pageIndex):
            if DEBUG_PRINT >= 1:
                logger.info(f"processOneDish is Empty")
            AsyncTaskLogger.log_task_end(task_id=pageID)
            return

        t1_processOneDish = time.time()
        dishOK = False
        retryIPTime = 3
        changeIP = False
        try:
            for i in range(retryIPTime):
                # change ip
                if DEBUG_PRINT >= 2:
                    t2_processOneDish = time.time()
                    msg = f'task_id={pageID} 1=== change IP{i} 次 changeIP = {changeIP} {pageUrl} time: {round(t2_processOneDish-t1_processOneDish, 3)}'
                    logger.debug(f"{msg}")
                contectPage = await self.contextPagePool.acquire(pageID, changeIP)
                if DEBUG_PRINT >= 2:
                    t3_processOneDish = time.time()
                    msg = f'task_id={pageID} 2=== end acquire ip {i} 次 changeIP={changeIP} {pageUrl} time: {round(t3_processOneDish-t2_processOneDish, 3)}'
                    logger.debug(f"{msg}")
                if await self.pageGo(pageID, contectPage['page'], pageUrl):#1.4s
                    if DEBUG_PRINT >= 2:
                        t4_processOneDish = time.time()
                        msg = f'task_id={pageID} 3=== end pageGo ip {i} 次 changeIP={changeIP} pageIndex = {contectPage["pageIndex"]} url={contectPage["page"].url} time: {round(t4_processOneDish - t3_processOneDish, 3)}'
                        logger.debug(f"{msg}")
                    onePageDishs = await self.extractOnePage(categoryIndex, contectPage['page'])
                    if DEBUG_PRINT >= 2:
                        t5_processOneDish = time.time()
                        msg = f'task_id={pageID} 4=== end extractOneDish ip {i} 次 changeIP={changeIP} pageIndex = {contectPage["pageIndex"]} url={contectPage["page"].url} time: {round(t5_processOneDish - t4_processOneDish, 3)}'
                        logger.debug(f"{msg}")
                    await self.contextPagePool.release(contectPage)
                    changeIP = False
                    await self.dump.openCsv(categoryIndex)
                    await self.dump.writeDishs([onePageDishs])
                    await self.dump.closeCsv()
                    if DEBUG_PRINT >= 2:
                        t6_processOneDish = time.time()
                        msg = f'task_id={pageID} 5=== end release ip {i} 次 changeIP={changeIP} pageIndex = {contectPage["pageIndex"]} url={contectPage["page"].url} time: {round(t6_processOneDish -t5_processOneDish, 3)}'
                        logger.debug(f"{msg}")
                    dishOK = True
                    break
                else:
                    await self.contextPagePool.release(contectPage)
                    changeIP = True
                    if DEBUG_PRINT >= 2:
                        t7_processOneDish = time.time()
                        msg = f'task_id={pageID} 6=== end release ip {i} 次 changeIP={changeIP} pageIndex = {contectPage["pageIndex"]} url={contectPage["page"].url} time: {round(t7_processOneDish-t3_processOneDish, 3)}'
                        logger.debug(f"{msg}")
                    continue
        except Exception as e:
            await self.contextPagePool.release(contectPage)
            if DEBUG_PRINT >= 1:
                t8_processOneDish = time.time()
                msg = f'task_id={pageID} 7=== end release ip {i} 次 changeIP={changeIP} pageIndex = {contectPage["pageIndex"]} url={contectPage["page"].url} time: {round(t8_processOneDish-t1_processOneDish, 3)}'
                AsyncTaskLogger.log_exception(e, pageID, msg)
            
        if DEBUG_PRINT >= 2:
            if dishOK:
                msg = f'task_id={pageID} 8=== DONE  {categoryIndex} {pageUrl}'
                logger.debug(f"{msg}")
            else:
                msg = f'task_id={pageID} 8=== Failed  {categoryIndex} {pageUrl}'
                logger.debug(f"{msg}")


        if DEBUG_PRINT >= 1:
            t9_processOneDish = time.time()
            logger.info(f"processOneDish Completed time: {round(t9_processOneDish-t1_processOneDish, 3)}")
            AsyncTaskLogger.log_task_end(task_id=pageID)
        await asyncio.sleep(REQUEST_INTERVAL)


    async def processOnePageWithLimit(self, categoryIndex, pageUrl):
        # async with async_timeout.timeout(30):
        pageID = categoryIndex + '_' + pageUrl.split('/')[-2]
        time_print(f"任务 {pageID} 等待 semaphore，当前可用: {self.semaphore._value}")
        
        async with self.semaphore:
            if DEBUG_PRINT>=2:
                time_print(f"任务 {pageID} 获得 semaphore，开始执行")
            try:
                result = await self.processOnePage(categoryIndex, pageUrl)
                if DEBUG_PRINT>=2:
                    time_print(f"任务 {pageID} 执行完成")
                return result
            except Exception as e:
                if DEBUG_PRINT>=1:
                    time_print(f"任务 {pageID} 执行异常: {type(e).__name__}: {e}")
                raise
            finally:
                if DEBUG_PRINT>=2:
                    time_print(f"任务 {pageID} 释放 semaphore，当前可用: {self.semaphore._value}")
                pass
        
        time_print(f"任务 {pageIndex} 已经释放后，semaphore，当前可用: {self.semaphore._value}")